{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "GPU_id = 1\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = str(GPU_id)\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from time import time \n",
    "\n",
    "from fastai import *\n",
    "from fastai.basic_data import *\n",
    "from fastai.basic_data import *\n",
    "from fastai.tabular import *\n",
    "from fastai.basic_data import DataBunch\n",
    "from fastai.tabular import TabularModel\n",
    "\n",
    "import cudf\n",
    "\n",
    "from preproc import *\n",
    "from batchloader import *\n",
    "from helpers import get_mean_reciprocal_rank, roc_auc_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- In this notebook we want to benchmark the processing and training time for three diffrent models: \n",
    "\n",
    "- The two first models are using our CuDF processing workflow <a href=#cudf_workflow> section I </a>:\n",
    "     1.  <a href=#first_model> Model 1 </a> : CuDF processing with CPU a copy\n",
    "     2.  <a href=#second_model> Model 2 </a> : CuDF processing in-memory without copy    \n",
    "\n",
    "           \n",
    " - <a href=#third_model> Model 3 </a> : In the second <a href=#fastai_workflow> section II </a>, we are using the Fastai processing workflow to get the scores of the best model found in the section I.  We directly process and create databunch from data_pair_all.pkl dataframe "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**N.B** : For each model, you need to re-start the kernel to free the GPU memory and be able to run all the experiments "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext snakeviz\n",
    "# load snakeviz if you want to run profiling "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> <center> <a id=batchdatabunch>New Data Bunch </a></center> </h1> "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define a custom databunch fastai that takes a TensorBatchDataLoader instead of the usual torch DataLoader "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BatchDataBunch(DataBunch):\n",
    "    \n",
    "    @classmethod\n",
    "    def remove_tfm(cls, tfm:Callable)->None:\n",
    "        \"Remove `tfm` from `self.tfms`.\"\n",
    "        if tfm in cls.tfms: cls.tfms.remove(tfm)\n",
    "            \n",
    "    @classmethod\n",
    "    def add_tfm(cls,tfm:Callable)->None:\n",
    "        \"Add `tfm` to `self.tfms`.\"\n",
    "        cls.tfms.append(tfm)\n",
    "\n",
    "    \n",
    "    @classmethod\n",
    "    def create(cls, train_ds, valid_ds, test_ds=None, path:PathOrStr='.', bs:int=64, val_bs=None, \n",
    "                      num_workers:int=defaults.cpus, device:torch.device=None,\n",
    "                      collate_fn:Callable=data_collate, tfms: List[Callable]=None, \n",
    "                       size:int=None, **kwargs)->'BatchDataBunch':\n",
    "        \n",
    "        \n",
    "        cls.tfms = listify(tfms)\n",
    "        \n",
    "        \n",
    "        val_bs = ifnone(val_bs, bs)\n",
    "        \n",
    "        datasets = [TensorBatchDataset(train_ds, batch_size=bs), \n",
    "                    TensorBatchDataset(valid_ds, batch_size=bs)]\n",
    "        \n",
    "        if valid_ds is not None:\n",
    "            cls.empty_val = False\n",
    "        else:\n",
    "            cls.empty_val = True\n",
    "            \n",
    "        if test_ds is not None:\n",
    "            datasets.append(TensorBatchDataset(test_ds, batch_size=bs))\n",
    "        else: \n",
    "            datasets.append(test_ds)\n",
    "        \n",
    "        cls.device = defaults.device if device is None else device\n",
    "        \n",
    "        dls = [BatchDataLoader(d, shuffle=s, pin_memory=False, drop_last=False, device=cls.device) for d,s in\n",
    "               zip(datasets,(True,False,False)) if d is not None]\n",
    "\n",
    "        cls.path = path \n",
    "        \n",
    "        cls.dls = dls\n",
    "    \n",
    "        \n",
    "        \n",
    "        assert not isinstance(dls[0],DeviceDataLoader)\n",
    "        \n",
    "        \n",
    "        # load batch in device \n",
    "        \n",
    "        if test_ds is not None:\n",
    "            cls.train_dl, cls.valid_dl, cls.test_dl = dls\n",
    "        else: \n",
    "            cls.train_dl, cls.valid_dl = dls\n",
    "            \n",
    "            \n",
    "        cls.path = Path(path)\n",
    "        return cls\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- To use the new BatchDatabunch class, we have to build the following processed tensors ( using cudf)  : \n",
    "    - train : cat_tensor, cont_tensor, label_tensor \n",
    "    \n",
    "    - valid : cat_tensor, cont_tensor, label_tensor \n",
    "    \n",
    "    - test : cat_tensor, cont_tensor, label_tensor \n",
    "    \n",
    "- The size of vocaublary of each categorical variable need to be known "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2> <a >Tabular Data in-memory  </a> </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_cpu = False "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Processing: Definition of train, validation and test tensors </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "read train used 7.86 seconds.\n",
      "get variables names used 0.00 seconds.\n",
      "processing train used 11.92 seconds.\n",
      "read test used 12.70 seconds.\n",
      "processing test used 9.36 seconds.\n",
      "read valid used 10.35 seconds.\n",
      "processing valid used 11.71 seconds.\n",
      "The whole processing used 42.63 seconds.\n"
     ]
    }
   ],
   "source": [
    "# %%snakeviz \n",
    "# uncomment the line above to generate the snakeviz profile of preprocessing \n",
    "\n",
    "data_path = './parquet_data/data_pair_all/'\n",
    "TEST = 'test'\n",
    "VALID = 'valid'\n",
    "TRAIN = 'train'\n",
    "\n",
    "start0 = time()\n",
    "data = {}\n",
    "\n",
    "############################\n",
    "#                          #\n",
    "# Fit processing train set #\n",
    "#                          #\n",
    "############################\n",
    "start = time()\n",
    "path = os.path.join(data_path,TRAIN+'.parquet' )\n",
    "ds = cudf.read_parquet(path)\n",
    "print(f\"read {TRAIN} used {time()-start:.2f} seconds.\")\n",
    "\n",
    "# get variable names \n",
    "start = time()\n",
    "cat_names = ['user_id','item_id','platform','city','device','current_filters'] + [i for i in ds.columns if i.startswith('is_')]\n",
    "cont_names = ['price','candidate_order'] + [i for i in ds.columns if i.startswith('count') or 'rank' in i or i.startswith('delta_')]\n",
    "print(f\"get variables names used {time()-start:.2f} seconds.\")\n",
    "\n",
    "# init the processing class \n",
    "proc = PreprocessDF(cat_names=cat_names, cont_names=cont_names, label_name='target', to_cpu=to_cpu)\n",
    "\n",
    "# Fit training \n",
    "start = time()\n",
    "x, y = proc.preproc_dataframe(ds, mode=TRAIN)\n",
    "print(f\"processing {TRAIN} used {time()-start:.2f} seconds.\")\n",
    "del ds\n",
    "data[TRAIN] = (x, y)\n",
    "\n",
    "############################\n",
    "#                          #\n",
    "# Transform test and valid #\n",
    "#                          #\n",
    "############################  \n",
    "ds_name = [TEST, VALID]\n",
    "for name in ds_name:\n",
    "    path = os.path.join(data_path,name+'.parquet' )\n",
    "    ds = cudf.read_parquet(path)\n",
    "\n",
    "    print(f\"read {name} used {time()-start:.2f} seconds.\")\n",
    "    start = time()\n",
    "    x, y = proc.preproc_dataframe(ds, mode=name)\n",
    "    print(f\"processing {name} used {time()-start:.2f} seconds.\")\n",
    "    data[name] = (x, y)\n",
    "    del ds\n",
    "\n",
    "print(f\"The whole processing used {time()-start0:.2f} seconds.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Benchmark : Get the best (batch size, learning rate)</h3> \n",
    "\n",
    "- Fine tune the best couple (batch_size, lr) : The criterion used is the CrossEntropy loss function \n",
    "    - The range of batch sizes is : 4096, 8192, 20480, 40960, 81920, 204800, 409600, 819200\n",
    "    - The range of max learning rate was set w.r.t to the plot of the results of the Fastai method find_lr : [6e-2, 9e-2, 2e-1] \n",
    "    \n",
    "    \n",
    "- **N.B:** Some of the batch_sizes require more than one epoch to get the best score (numbers shown in the paper). However, to the complexity of the notebook, we'll run all the training with 1 epoch as our best model (fastest training time) converges in 1 epoch. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_sizes = [4096, 8192, 20480, 40960, 81920, 204800, 409600, 819200]\n",
    "lrs = [6e-2, 9e-2, 2e-1] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.113469</td>\n",
       "      <td>0.117696</td>\n",
       "      <td>02:56</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.06, bs: 4096 used 176.37\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.113058</td>\n",
       "      <td>0.116693</td>\n",
       "      <td>03:01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.09, bs: 4096 used 181.84\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.115227</td>\n",
       "      <td>0.120717</td>\n",
       "      <td>02:53</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.2, bs: 4096 used 173.59\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.111515</td>\n",
       "      <td>0.117041</td>\n",
       "      <td>02:23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.06, bs: 8192 used 143.79\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.111174</td>\n",
       "      <td>0.116100</td>\n",
       "      <td>02:27</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.09, bs: 8192 used 147.70\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.114644</td>\n",
       "      <td>0.117820</td>\n",
       "      <td>02:23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.2, bs: 8192 used 143.74\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.109096</td>\n",
       "      <td>0.118919</td>\n",
       "      <td>02:13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.06, bs: 20480 used 133.46\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.110107</td>\n",
       "      <td>0.116204</td>\n",
       "      <td>02:14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.09, bs: 20480 used 134.21\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.110983</td>\n",
       "      <td>0.452330</td>\n",
       "      <td>02:13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.2, bs: 20480 used 133.54\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.110156</td>\n",
       "      <td>0.115422</td>\n",
       "      <td>02:16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.06, bs: 40960 used 136.28\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.110243</td>\n",
       "      <td>0.116298</td>\n",
       "      <td>02:15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.09, bs: 40960 used 135.80\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.110850</td>\n",
       "      <td>0.132129</td>\n",
       "      <td>02:15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.2, bs: 40960 used 135.39\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.112239</td>\n",
       "      <td>0.115680</td>\n",
       "      <td>02:18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.06, bs: 81920 used 138.78\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.112199</td>\n",
       "      <td>0.115370</td>\n",
       "      <td>02:18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.09, bs: 81920 used 138.36\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.113172</td>\n",
       "      <td>0.116391</td>\n",
       "      <td>02:18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.2, bs: 81920 used 138.35\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.125682</td>\n",
       "      <td>0.117607</td>\n",
       "      <td>01:37</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.06, bs: 204800 used 97.95\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.123097</td>\n",
       "      <td>0.117060</td>\n",
       "      <td>01:37</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.09, bs: 204800 used 97.68\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.121157</td>\n",
       "      <td>0.117040</td>\n",
       "      <td>01:38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.2, bs: 204800 used 98.01\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.165766</td>\n",
       "      <td>0.123385</td>\n",
       "      <td>01:46</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.06, bs: 409600 used 106.95\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.159684</td>\n",
       "      <td>0.126204</td>\n",
       "      <td>01:46</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.09, bs: 409600 used 106.64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.151141</td>\n",
       "      <td>0.123508</td>\n",
       "      <td>01:46</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.2, bs: 409600 used 106.78\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.199256</td>\n",
       "      <td>0.130854</td>\n",
       "      <td>01:48</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.06, bs: 819200 used 108.96\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.195795</td>\n",
       "      <td>0.131586</td>\n",
       "      <td>01:48</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.09, bs: 819200 used 108.70\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.191146</td>\n",
       "      <td>0.136116</td>\n",
       "      <td>01:48</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training for the couple: lr: 0.2, bs: 819200 used 108.75\n"
     ]
    }
   ],
   "source": [
    "# Define batch databunch \n",
    "from fastai.tabular import TabularModel\n",
    "from time import time \n",
    "benchmark_results = [] \n",
    "\n",
    "for batch_size in batch_sizes: \n",
    "    train = [data['train'][0][0], data['train'][0][1], data['train'][1].long()]\n",
    "    validation = [data['valid'][0][0], data['valid'][0][1], data['valid'][1].long()]\n",
    "    test = [data['test'][0][0], data['test'][0][1], data['test'][1].long()]\n",
    "    databunch = BatchDataBunch.create(train, validation, device='cuda', bs=batch_size)   \n",
    "    del train \n",
    "    del validation \n",
    "    del test\n",
    "    for learning_rate in lrs: \n",
    "        #define the model \n",
    "        \n",
    "        emb_sz = [(938604, 16), (903867, 16), (56, 4), (32763, 8), (4, 1), (27842, 8), \n",
    "                  (3, 3), (3, 3), (3, 3), (3, 3), (3, 3), (3, 3), (3, 3)]  \n",
    "\n",
    "\n",
    "        model = TabularModel(emb_szs = emb_sz, n_cont=len(cont_names), out_sz=2, layers=[64, 32])\n",
    "\n",
    "        learn =  Learner(databunch, model, metrics=None)\n",
    "        \n",
    "        learn.loss_func = torch.nn.CrossEntropyLoss()\n",
    "        \n",
    "        # launch training \n",
    "        start = time()\n",
    "        learn.fit_one_cycle(1, learning_rate)\n",
    "        t_final = time() - start \n",
    "        benchmark_results.append([batch_size, learning_rate, learn.recorder.val_losses[0], 1, t_final] ) \n",
    "        print('training for the couple: lr: %s, bs: %s used %.2f' %(learning_rate, batch_size, t_final))\n",
    "\n",
    "        del learn \n",
    "        del model\n",
    "        torch.cuda.empty_cache()   \n",
    "        \n",
    "    del databunch\n",
    "    torch.cuda.empty_cache()   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(benchmark_results)\n",
    "results.columns = ['batch size', 'learning rate', 'validation loss', 'N epochs', 'training time']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>batch size</th>\n",
       "      <th>learning rate</th>\n",
       "      <th>validation loss</th>\n",
       "      <th>N epochs</th>\n",
       "      <th>training time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>204800</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.117060</td>\n",
       "      <td>1</td>\n",
       "      <td>97.675189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>204800</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.117607</td>\n",
       "      <td>1</td>\n",
       "      <td>97.948497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>204800</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.117040</td>\n",
       "      <td>1</td>\n",
       "      <td>98.009450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>409600</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.126204</td>\n",
       "      <td>1</td>\n",
       "      <td>106.638807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>409600</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.123508</td>\n",
       "      <td>1</td>\n",
       "      <td>106.780997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>409600</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.123385</td>\n",
       "      <td>1</td>\n",
       "      <td>106.953093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>819200</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.131586</td>\n",
       "      <td>1</td>\n",
       "      <td>108.699999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>819200</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.136116</td>\n",
       "      <td>1</td>\n",
       "      <td>108.745657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>819200</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.130854</td>\n",
       "      <td>1</td>\n",
       "      <td>108.957399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>20480</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.118919</td>\n",
       "      <td>1</td>\n",
       "      <td>133.462606</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    batch size  learning rate  validation loss  N epochs  training time\n",
       "16      204800           0.09         0.117060         1      97.675189\n",
       "15      204800           0.06         0.117607         1      97.948497\n",
       "17      204800           0.20         0.117040         1      98.009450\n",
       "19      409600           0.09         0.126204         1     106.638807\n",
       "20      409600           0.20         0.123508         1     106.780997\n",
       "18      409600           0.06         0.123385         1     106.953093\n",
       "22      819200           0.09         0.131586         1     108.699999\n",
       "23      819200           0.20         0.136116         1     108.745657\n",
       "21      819200           0.06         0.130854         1     108.957399\n",
       "6        20480           0.06         0.118919         1     133.462606"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.sort_values(by=['training time', 'validation loss'], ascending=True).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Conclusion** The best trade-off between training time and validation loss is reached for the couple **(204800, 0.09)**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Compute average validation scores of the best model </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from helpers import get_mean_reciprocal_rank, roc_auc_score\n",
    "path = os.path.join(data_path,'valid.parquet' )\n",
    "ds = pd.read_parquet(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Mean / std of scores : 5 runs \n",
    "aucs = []\n",
    "mrrs = []\n",
    "times = []\n",
    "best_bs = 4096*50\n",
    "best_lr = 9e-2\n",
    "\n",
    "# create databunch \n",
    "train = [data['train'][0][0], data['train'][0][1], data['train'][1].long()]\n",
    "validation = [data['valid'][0][0], data['valid'][0][1], data['valid'][1].long()]\n",
    "test = [data['test'][0][0], data['test'][0][1], data['test'][1].long()]\n",
    "databunch = BatchDataBunch.create(train, validation, device='cuda', bs=best_bs)   \n",
    "\n",
    "del train \n",
    "del validation \n",
    "del test\n",
    "\n",
    "emb_sz = [(938604, 16), (903867, 16), (56, 4), (32763, 8), (4, 1), (27842, 8), \n",
    "          (3, 3), (3, 3), (3, 3), (3, 3), (3, 3), (3, 3), (3, 3)]  \n",
    "\n",
    "\n",
    "# run the model 5 times to get 5 scores \n",
    "\n",
    "for i in range(5): \n",
    "    # define the model\n",
    "    model = TabularModel(emb_szs = emb_sz, n_cont=len(cont_names), out_sz=2, layers=[64, 32])\n",
    "    model = model.cuda()\n",
    "    learn =  Learner(databunch, model, metrics=None)\n",
    "    learn.loss_func = torch.nn.CrossEntropyLoss()\n",
    "    \n",
    "    # train the model \n",
    "    start = time()\n",
    "    learn.fit_one_cycle(1, best_lr)\n",
    "    tf = time()-start\n",
    "    \n",
    "    # get validation metrics \n",
    "    yp,y_valid = learn.get_preds(databunch)\n",
    "    cv = ds[['row_id','reference','item_id', 'target']].copy()\n",
    "    cv['prob'] = yp.numpy()[:,1]\n",
    "    cv = cv.sort_values(by=['row_id','prob'],ascending=False)\n",
    "    auc = roc_auc_score(y_valid.numpy().ravel(),yp.numpy()[:,1])\n",
    "    mean_reciprocal_rank = get_mean_reciprocal_rank(cv)\n",
    "    \n",
    "    aucs.append(auc)\n",
    "    mrrs.append(mean_reciprocal_rank)\n",
    "    times.append(tf)\n",
    "    \n",
    "    del model \n",
    "    del learn\n",
    "    torch.cuda.empty_cache()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the mrr of the best mdodel is: 0.611057369826576 +/- 0.0012801593160030626\n",
      "the auc of the best mdodel is: 0.8766171778021861 +/- 0.004299151814914718\n",
      "the best mdodel's training time is 97.94034910202026 +/- 0.126405132714989\n"
     ]
    }
   ],
   "source": [
    "print(\"the mrr of the best mdodel is: %s +/- %s\" %(np.mean(mrrs), np.std(mrrs)))\n",
    "\n",
    "print(\"the auc of the best mdodel is: %s +/- %s\" %(np.mean(aucs), np.std(aucs)))\n",
    "\n",
    "print(\"the best mdodel's training time is %s +/- %s\" %(np.mean(times), np.std(times)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
